{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "df = pd.read_csv(\"new_data.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   millisecond                                               hash  \\\n",
      "0            0  42fb5e2ec009a05ff5143227297074f1e9c6c3ebb9c914...   \n",
      "1            1  42fb5e2ec009a05ff5143227297074f1e9c6c3ebb9c914...   \n",
      "2            2  42fb5e2ec009a05ff5143227297074f1e9c6c3ebb9c914...   \n",
      "3            3  42fb5e2ec009a05ff5143227297074f1e9c6c3ebb9c914...   \n",
      "4            4  42fb5e2ec009a05ff5143227297074f1e9c6c3ebb9c914...   \n",
      "\n",
      "  classification  state  usage_counter        prio  static_prio  normal_prio  \\\n",
      "0        malware      0              0  3069378560        14274            0   \n",
      "1        malware      0              0  3069378560        14274            0   \n",
      "2        malware      0              0  3069378560        14274            0   \n",
      "3        malware      0              0  3069378560        14274            0   \n",
      "4        malware      0              0  3069378560        14274            0   \n",
      "\n",
      "   policy  vm_pgoff      ...       nivcsw  min_flt  maj_flt  fs_excl_counter  \\\n",
      "0       0         0      ...            0        0      120                0   \n",
      "1       0         0      ...            0        0      120                0   \n",
      "2       0         0      ...            0        0      120                0   \n",
      "3       0         0      ...            0        0      120                0   \n",
      "4       0         0      ...            0        0      120                0   \n",
      "\n",
      "         lock   utime  stime  gtime  cgtime  signal_nvcsw  \n",
      "0  3204448256  380690      4      0       0             0  \n",
      "1  3204448256  380690      4      0       0             0  \n",
      "2  3204448256  380690      4      0       0             0  \n",
      "3  3204448256  380690      4      0       0             0  \n",
      "4  3204448256  380690      4      0       0             0  \n",
      "\n",
      "[5 rows x 35 columns]\n"
     ]
    }
   ],
   "source": [
    "print(df.head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['millisecond',\n",
       " 'hash',\n",
       " 'classification',\n",
       " 'state',\n",
       " 'usage_counter',\n",
       " 'prio',\n",
       " 'static_prio',\n",
       " 'normal_prio',\n",
       " 'policy',\n",
       " 'vm_pgoff',\n",
       " 'vm_truncate_count',\n",
       " 'task_size',\n",
       " 'cached_hole_size',\n",
       " 'free_area_cache',\n",
       " 'mm_users',\n",
       " 'map_count',\n",
       " 'hiwater_rss',\n",
       " 'total_vm',\n",
       " 'shared_vm',\n",
       " 'exec_vm',\n",
       " 'reserved_vm',\n",
       " 'nr_ptes',\n",
       " 'end_data',\n",
       " 'last_interval',\n",
       " 'nvcsw',\n",
       " 'nivcsw',\n",
       " 'min_flt',\n",
       " 'maj_flt',\n",
       " 'fs_excl_counter',\n",
       " 'lock',\n",
       " 'utime',\n",
       " 'stime',\n",
       " 'gtime',\n",
       " 'cgtime',\n",
       " 'signal_nvcsw']"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(df.columns.values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "com.zynga.wordsontour.apk                                           1000\n",
      "35e61d9b00a30f757d9b96fe5e5e2c89a8ebaa6e5787eb4b7e3a4a4213c4ce97    1000\n",
      "com.jrtstudio.music.apk                                             1000\n",
      "com.google.android.apps.docs.editors.docs.apk                       1000\n",
      "com.ezmusicplayer.demo.apk                                          1000\n",
      "2f14c6999815d6a802a5106ed898554c22a6592ca37bd90fda560d46e3123892    1000\n",
      "84892f7a0b371c835ad31d4462222646a610925873961084de2ebd8486d478ad    1000\n",
      "biz.mtoy.blockpuzzle.revolution.apk                                 1000\n",
      "com.king.candycrushsodasaga.apk                                     1000\n",
      "12fbe832590c8d44b1687b178450d49190c1a9d8e61c80a090f461168dd0bf8f    1000\n",
      "68366d9e4a9099fa88f733dad4f5e1731e22cedfe7742c2363a46613ea8a1309    1000\n",
      "com.androiddevelopermx.blogspot.organos3d.apk                       1000\n",
      "2c9adaa2f7a5abad259da852850d2e9296f737a84517d29a45c0baed0589c613    1000\n",
      "079277b8b6049c06806b79216901d0e9ff473bfe2c2454aa8a496515167eca40    1000\n",
      "7590e4a832b9a17bdea9904cc84d8d132ce8b218b7e7e40086509d5f6f728ac7    1000\n",
      "com.bti.myGuitar.apk                                                1000\n",
      "368342ac090a20c9eaa3e4c40f701ee19aabb9fb1177c63b1ff8adbf4947b934    1000\n",
      "com.medicaljoyworks.prognosis.apk                                   1000\n",
      "com.modernenglishstudio.HowToSpeak.apk                              1000\n",
      "com.tyengl.vocab.apk                                                1000\n",
      "com.fingerprintplay.bysbaseball2015.apk                             1000\n",
      "com.vbsmojivy.mianzed.apk                                           1000\n",
      "com.sonypicturestelevision.sportsjeopardy.apk                       1000\n",
      "42fb5e2ec009a05ff5143227297074f1e9c6c3ebb9c914e223349672eca79ad0    1000\n",
      "8068747ccbbddb0e9596fd5dad9ac39042ff5c01a8c2ad55644783baa314f7d4    1000\n",
      "025c63d266e05d9e3bd57dd9ebd0abe904616f569fe4e2b78cf2ac52493cb460    1000\n",
      "air.com.tensquaregames.letsfish.apk                                 1000\n",
      "com.kmcpesh.medicalskillsproceduresfree.apk                         1000\n",
      "com.baiwang.instablend.apk                                          1000\n",
      "com.epocrates.apk                                                   1000\n",
      "                                                                    ... \n",
      "2e4c54588cca3be3ae471b4b9b531ed2a70b2d336688fef6d5bbed7ba21db580    1000\n",
      "com.rottzgames.realjigsaw.apk                                       1000\n",
      "air.com.freshplanet.games.MoviePop.apk                              1000\n",
      "6f158980b71ae3d8ffc462fa6c63ca0a3cc0a7ab7cc079bede1279c4f67fae23    1000\n",
      "32effc5a6bc3b7319b5b7da02a7cc3576d44c1794b335be71ab8f3545a0555bc    1000\n",
      "2f7693ee9f8a349d6c8f4e1a90a9c6a41b774d48c17aad205d3dfc799a4d74a3    1000\n",
      "711415bfe471619f1dd4dbdae0a9d82f5c01b44f57501d59f1d37479411b50c4    1000\n",
      "com.piriform.ccleaner.apk                                           1000\n",
      "com.fitnesskeeper.runkeeper.pro.apk                                 1000\n",
      "21b41814b140cd9824b72e765270800f7975fa4c8955396a95370567d95138ae    1000\n",
      "com.i6.FlightSimulatorAirplane3D.apk                                1000\n",
      "3c722b32535e6e8ea1bbf8accdff73376cf2c3393d9325304ffc37c3213fdb4c    1000\n",
      "com.zayaninfotech.english.grammar.apk                               1000\n",
      "3d51872172186d55238444384224ca46d0f1a7ab87910494c6354a7a53074387    1000\n",
      "air.com.KalromSystems.BestButtFitness.apk                           1000\n",
      "68da8872328d2c6eeec52ff92e603949a8dfc50166d91491903d9d846afa1eeb    1000\n",
      "1dec265aeda7b58e4173f47af0641a949937edbf21904ff1b6681c5348642387    1000\n",
      "5dd6c684ad85ec01c32172a38451f0b6f3b261dce3c335dbe099d87763fe7790    1000\n",
      "116ae92ecfacb70146fe643d92878e522f71af393702f3b66d2135a06bcff57f    1000\n",
      "186d3233e77f4a0c64043da385fb7f0dcd195ee0c1f46d3e8f49d4bf8d5d2d1f    1000\n",
      "DOCECG2.doctor.apk                                                  1000\n",
      "com.mobilityware.solitaire.apk                                      1000\n",
      "com.google.zxing.client.android.apk                                 1000\n",
      "54b860b1c538d915a68a1980afd25c029458ade80e4175e6487caad9b17c7e99    1000\n",
      "com.magmamobile.game.Words.apk                                      1000\n",
      "com.music.choice.apk                                                1000\n",
      "com.miniclip.dudeperfect.apk                                        1000\n",
      "com.microsoft.amp.apps.bingfinance.apk                              1000\n",
      "1efc135b8f924076b56564ee93f5a82794671e0f0da08b1b5ccd9ac048c5fda5    1000\n",
      "4fd8fb06b479ae910df2e8806d3a6968a921c9cad596cc06aadd5c7e04c7df2f    1000\n",
      "Name: hash, Length: 100, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "print(df['hash'].value_counts())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0    100000\n",
      "Name: usage_counter, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "print(df['usage_counter'].value_counts())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "benign     50000\n",
      "malware    50000\n",
      "Name: classification, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "print(df['classification'].value_counts())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7214     1030\n",
      "6925     1007\n",
      "6137     1006\n",
      "6768     1002\n",
      "4105     1001\n",
      "4247     1000\n",
      "3748     1000\n",
      "5687     1000\n",
      "2588     1000\n",
      "8294      988\n",
      "7968      908\n",
      "2666      813\n",
      "7321      808\n",
      "10364     779\n",
      "2686      755\n",
      "2664      739\n",
      "2657      718\n",
      "11113     667\n",
      "3720      667\n",
      "7490      666\n",
      "7435      656\n",
      "2655      639\n",
      "8238      637\n",
      "7097      604\n",
      "9747      588\n",
      "7865      568\n",
      "10684     565\n",
      "6495      554\n",
      "14415     551\n",
      "7900      544\n",
      "         ... \n",
      "13484       1\n",
      "13972       1\n",
      "21728       1\n",
      "11863       1\n",
      "27879       1\n",
      "5352        1\n",
      "3667        1\n",
      "19995       1\n",
      "24089       1\n",
      "13548       1\n",
      "9454        1\n",
      "5648        1\n",
      "21976       1\n",
      "5480        1\n",
      "5608        1\n",
      "7529        1\n",
      "26014       1\n",
      "15725       1\n",
      "9622        1\n",
      "9582        1\n",
      "3475        1\n",
      "17754       1\n",
      "11607       1\n",
      "9558        1\n",
      "15701       1\n",
      "7593        1\n",
      "13740       1\n",
      "11543       1\n",
      "17890       1\n",
      "4094        1\n",
      "Name: map_count, Length: 10578, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "print(df['map_count'].value_counts())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0           60933\n",
      "4096        14757\n",
      "8192         4133\n",
      "12288        2714\n",
      "1007616      1021\n",
      "1044480       997\n",
      "1032192       881\n",
      "16384         805\n",
      "24576         761\n",
      "1040384       678\n",
      "114688        590\n",
      "466944        499\n",
      "45056         498\n",
      "516096        490\n",
      "20480         404\n",
      "61440         367\n",
      "36864         367\n",
      "32768         334\n",
      "933888        309\n",
      "1093632       294\n",
      "28672         285\n",
      "2088960       281\n",
      "53248         271\n",
      "479232        248\n",
      "917504        237\n",
      "2277376       236\n",
      "1024000       232\n",
      "258048        194\n",
      "1048576       192\n",
      "11415552      189\n",
      "            ...  \n",
      "4132864         1\n",
      "3153920         1\n",
      "7860224         1\n",
      "151552          1\n",
      "2883584         1\n",
      "43266048        1\n",
      "905216          1\n",
      "1458176         1\n",
      "167936          1\n",
      "3866624         1\n",
      "770048          1\n",
      "749568          1\n",
      "6160384         1\n",
      "2703360         1\n",
      "3014656         1\n",
      "892928          1\n",
      "12075008        1\n",
      "1998848         1\n",
      "12038144        1\n",
      "4280320         1\n",
      "552960          1\n",
      "696320          1\n",
      "1953792         1\n",
      "6807552         1\n",
      "2408448         1\n",
      "6754304         1\n",
      "159744          1\n",
      "999424          1\n",
      "1011712         1\n",
      "634880          1\n",
      "Name: state, Length: 304, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "print(df['state'].value_counts())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Basic data cleaning"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Handling missing data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "signal_nvcsw        0\n",
       "policy              0\n",
       "mm_users            0\n",
       "free_area_cache     0\n",
       "cached_hole_size    0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.isnull().sum().sort_values(ascending=False).head()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
